"""
高频因子低频化：tick大单分布分析
使用大单市值来衡量大单 ，以500W为步进，从0~3000W进行分档
统计各个合约日内大单的分析
将多日联合形成日频数据

数据取自回测用的tick
"""
import os
import pandas as pd
import zipfile as zf
from vnpy.gateway.ctp.gateway_mapping import multiplier_mapping, price_tick_mapping, exchange_mapping
from vnpy.trader.utility import extract_sec_id


def analyse_single_file(tick_df, sec_size):
    if "trading" in tick_df.columns:
        tick_df = tick_df.loc[tick_df["trading"] == True]
    tick_df["tick_volume"] = tick_df["volume"] - tick_df["volume"].shift(1)
    tick_df.fillna(0, inplace=True)
    tick_df["market_value"] = tick_df["tick_volume"] * tick_df["last_price"] * sec_size / 5000000
    tick_df["market_value"] = tick_df["market_value"].astype(int)
    grouped = tick_df.groupby("market_value")
    counts = tick_df.shape[0]
    total_volume = tick_df.iloc[-1]["volume"]
    volume_dist = grouped["tick_volume"].sum() / total_volume
    counts_dist = grouped["tick_volume"].count() / counts
    return volume_dist, counts_dist

def analyse_all_file():
    tick_data_path = "D:\自采tick"
    tick_temp_path = "d:\\temp"
    all_files = os.listdir(tick_data_path)
    # all_files = all_files[:10]
    symbol_result_volumes = {}
    symbol_result_counts = {}
    for file_name in all_files:
        print(file_name)
        tick_data_zip = zf.ZipFile(f"{tick_data_path}\\{file_name}", mode="r")
        tick_data_zip.extractall(tick_temp_path)
        all_csv_files = os.listdir(tick_temp_path)
        dt, _ = file_name.split(".")
        for csv_file_name in all_csv_files:
            print(file_name, csv_file_name)
            if "bar" in csv_file_name:
                continue
            if "99" in csv_file_name:
                continue
            if "&" in csv_file_name:
                continue
            _, sec_name = csv_file_name.split("_", 1)
            sec_id = extract_sec_id(sec_name)
            symbol, _ = sec_name.split(".", 1)
            sec_size = multiplier_mapping[sec_id]
            try:
                df = pd.read_csv(f"{tick_temp_path}\\{csv_file_name}", encoding="GBK")
            except UnicodeDecodeError:
                df = pd.read_csv(f"{tick_temp_path}\\{csv_file_name}")
            volume_dist, counts_dist = analyse_single_file(df, sec_size)
            symbol_result_volume = symbol_result_volumes.setdefault(symbol, {})
            symbol_result_count = symbol_result_counts.setdefault(symbol, {})
            symbol_result_volume[dt] = volume_dist
            symbol_result_count[dt] = counts_dist
        for csv_file_name in all_csv_files:
            os.remove(f"{tick_temp_path}\\{csv_file_name}")
    for symbol, symbol_data in symbol_result_volumes.items():
        symbol_df = pd.DataFrame.from_dict(symbol_data, orient="index")
        symbol_df.to_csv(f"d:\\tick_volume_distribut\\{symbol}_volume.csv")
    for symbol, symbol_data in symbol_result_counts.items():
        symbol_df = pd.DataFrame.from_dict(symbol_data, orient="index")
        symbol_df.to_csv(f"d:\\tick_count_distribut\\{symbol}_count.csv")


def analyse_ask_bid(date, symbol, tick_df):
    """
    取分主动买和主动卖，分别统计：
        主动买的条件：最新成交价上涨
        主动卖的条件：最新成交价下跌
        最新成交价不变：
        - 买卖价不变或是同时扩大：
        - 最新成交价在范围外才算主动买卖
        - 买价变大：主动买
        - 买价变小：主动卖
    """
    # 过滤无效tick
    if "trading" in tick_df.columns:
        tick_df = tick_df.loc[tick_df["trading"] == True]
    # 标识主动买卖ask_bid
    tick_df["l1"] = tick_df["last_price"].shift(1)
    tick_df["ask_bid"] = "None"
    tick_df.loc[tick_df["last_price"] > tick_df["l1"], "ask_bid"] = "bid"
    tick_df.loc[tick_df["last_price"] < tick_df["l1"], "ask_bid"] = "ask"
    tick_df["bid_l1"] = tick_df["bid_price_1"].shift(1)
    tick_df["ask_l1"] = tick_df["ask_price_1"].shift(1)
    tick_df.loc[
        (tick_df["last_price"] == tick_df["l1"]) & (tick_df["last_price"] <= tick_df["bid_l1"]), "ask_bid"] = "ask"
    tick_df.loc[
        (tick_df["last_price"] == tick_df["l1"]) & (tick_df["last_price"] >= tick_df["ask_l1"]), "ask_bid"] = "bid"
    # 标识开平仓：仓位不减少即为开仓
    tick_df["p1"] = tick_df["open_interest"].shift(1)
    tick_df["oc"] = "close"
    tick_df.loc[tick_df["open_interest"] >= tick_df["p1"], "oc"] = "open"
    # 计算成交量
    tick_df["tick_volume"] = tick_df["volume"] - tick_df["volume"].shift(1)
    tick_df.fillna(0, inplace=True)
    # 总大单，先排序再排后20%
    tick_df.sort_values("tick_volume", inplace=True)
    tick_num = tick_df.shape[0]
    if tick_num < 3000:
        return None, None
    big_tick_num = int(tick_num * 0.2)
    tick_df["big_tick"] = 0
    big_tick_volume = tick_df.iloc[-big_tick_num]["tick_volume"]
    tick_df.loc[tick_df["tick_volume"] >= big_tick_volume, "big_tick"] = 1
    # 买top20大单
    bid_num = tick_df.loc[tick_df["ask_bid"] == "bid"].shape[0]
    big_bid_num = int(bid_num * 0.2)
    tick_df["big_bid"] = 0
    if big_bid_num:
        big_bid_volume = tick_df[tick_df["ask_bid"] == "bid"].iloc[-big_bid_num]["tick_volume"]
        tick_df.loc[(tick_df["ask_bid"] == "bid") & (tick_df["tick_volume"] >= big_bid_volume), "big_bid"] = 1
    # 卖top20大单
    ask_num = tick_df.loc[tick_df["ask_bid"] == "ask"].shape[0]
    big_ask_num = int(ask_num * 0.2)
    tick_df["big_ask"] = 0
    if big_ask_num:
        big_ask_volume = tick_df[tick_df["ask_bid"] == "ask"].iloc[-big_ask_num]["tick_volume"]
        tick_df.loc[(tick_df["ask_bid"] == "ask") & (tick_df["tick_volume"] >= big_ask_volume), "big_ask"] = 1
    # 变回时间排序
    tick_df.sort_index(inplace=True)
    # 一些计算量
    total_volume = tick_df.iloc[-1]["volume"]   # 总成交量
    total_big_volume = tick_df.loc[tick_df["big_tick"] == 1, "tick_volume"].sum()   # 大单量
    total_bid_volume = tick_df.loc[tick_df["ask_bid"] == "bid", "tick_volume"].sum()    # 总买量
    total_ask_volume = tick_df.loc[tick_df["ask_bid"] == "ask", "tick_volume"].sum()    # 总卖量
    # 买大单计算方式一：使用全量大单标准
    total_big_bid_volume = tick_df.loc[
        (tick_df["ask_bid"] == "bid") & (tick_df["big_tick"] == 1), "tick_volume"].sum()  # 大单买量
    total_big_bid_open_volume = tick_df.loc[
        (tick_df["ask_bid"] == "bid") &
        (tick_df["big_tick"] == 1) &
        (tick_df["oc"] == "open"), "tick_volume"].sum()  # 大单买开量
    # 买大单计算方式二：使用买大单标准
    total_big_bid_volume2 = tick_df.loc[
        (tick_df["ask_bid"] == "bid") & (tick_df["big_bid"] == 1), "tick_volume"].sum()  # 大单买量
    total_big_bid_open_volume2 = tick_df.loc[
        (tick_df["ask_bid"] == "bid") &
        (tick_df["big_bid"] == 1) &
        (tick_df["oc"] == "open"), "tick_volume"].sum()  # 大单买开量
    # 卖大单计算方式一：使用全量大单标准
    total_big_ask_volume = tick_df.loc[
        (tick_df["ask_bid"] == "ask") & (tick_df["big_tick"] == 1), "tick_volume"].sum()  # 大单卖量
    total_big_ask_open_volume = tick_df.loc[
        (tick_df["ask_bid"] == "ask") &
        (tick_df["big_tick"] == 1) &
        (tick_df["oc"] == "open"), "tick_volume"].sum()  # 大单卖开量
    # 卖大单计算方式二：使用卖大单标准
    total_big_ask_volume2 = tick_df.loc[
        (tick_df["ask_bid"] == "ask") & (tick_df["big_ask"] == 1), "tick_volume"].sum()  # 大单卖量
    total_big_ask_open_volume2 = tick_df.loc[
        (tick_df["ask_bid"] == "ask") &
        (tick_df["big_ask"] == 1) &
        (tick_df["oc"] == "open"), "tick_volume"].sum()  # 大单卖开量

    average_big = tick_df.loc[tick_df["big_tick"] == 1, "tick_volume"].mean()  # 大平均量
    average_big_bid = tick_df.loc[
        (tick_df["ask_bid"] == "bid") & (tick_df["big_tick"] == 1), "tick_volume"].mean()  # 大单买平均量
    average_big_ask = tick_df.loc[
        (tick_df["ask_bid"] == "ask") & (tick_df["big_tick"] == 1), "tick_volume"].mean()  # 大单卖平均量
    average_big_bid2 = tick_df.loc[
        (tick_df["ask_bid"] == "bid") & (tick_df["big_bid"] == 1), "tick_volume"].mean()  # 大单买平均量
    average_big_ask2 = tick_df.loc[
        (tick_df["ask_bid"] == "ask") & (tick_df["big_ask"] == 1), "tick_volume"].mean()  # 大单卖平均量

    # 以全量top20为标准统计结果
    result = {
        "date": date,
        "symbol": symbol,
        "Da_Z": 0 if not total_volume else total_big_volume / total_volume,     # 总大单量/总量
        "Buy_Z": 0 if not total_volume else total_big_bid_volume / total_volume,     # 总大单买量/总量
        "Buy_ZBuy": 0 if not total_bid_volume else total_big_bid_volume / total_bid_volume,   # 大主动买/总主动买
        "BuyOpen_ZBuy": 0 if not total_bid_volume else total_big_bid_open_volume / total_bid_volume,     # 大主动买开/总主动买
        "Sell_Z": 0 if not total_volume else total_big_ask_volume / total_volume,      # 总大单卖量/总量
        "Sell_ZSell": 0 if not total_ask_volume else total_big_ask_volume / total_ask_volume,   # 总大单卖量/总卖量
        "SellOpen_ZSell": 0 if not total_ask_volume else total_big_ask_open_volume / total_ask_volume,     # 总大单主动卖开量/总卖量
        "ZBuy_ZSell": 0 if not total_ask_volume else total_bid_volume / total_ask_volume,  # 总买/总卖
        "Buy_Sell": 0 if not total_big_ask_volume else total_big_bid_volume / total_big_ask_volume,    # 大单买/大单卖
        "BuyOpen_SellOpen": 0 if not total_big_ask_open_volume else total_big_bid_open_volume / total_big_ask_open_volume,   # 大单买开/大单卖开
        "Big_Average": average_big,
        "Buy_Average": average_big_bid,     # 全量top20里平均买大单量
        "Sell_Average": average_big_ask     # 全量top20里平均卖大单量
    }
    # 以买卖各自top20为标准统计结果
    result2 = {
        "date": date,
        "symbol": symbol,
        "Da_Z": 0 if not total_volume else total_big_volume / total_volume,     # 总大单量/总量
        "Buy_Z_2": 0 if not total_volume else total_big_bid_volume2 / total_volume,     # 总大单买量/总量
        "Buy_ZBuy_2": 0 if not total_bid_volume else total_big_bid_volume2 / total_bid_volume,   # 大主动买/总主动买
        "BuyOpen_ZBuy_2": 0 if not total_bid_volume else total_big_bid_open_volume2 / total_bid_volume,     # 大主动买开/总主动买
        "Sell_Z_2": 0 if not total_volume else total_big_ask_volume2 / total_volume,      # 总大单卖量/总量
        "Sell_ZSell_2": 0 if not total_ask_volume else total_big_ask_volume2 / total_ask_volume,   # 总大单卖量/总卖量
        "SellOpen_ZSell_2": 0 if not total_ask_volume else total_big_ask_open_volume2 / total_ask_volume,     # 总大单主动卖开量/总卖量
        "ZBuy_ZSell": 0 if not total_ask_volume else total_bid_volume / total_ask_volume,  # 总买/总卖
        "Buy_Sell_2": 0 if not total_big_ask_volume2 else total_big_bid_volume2 / total_big_ask_volume2,    # 大单买/大单卖
        "BuyOpen_SellOpen_2": 0 if not total_big_ask_open_volume2 else total_big_bid_open_volume2 / total_big_ask_open_volume2,   # 大单买开/大单卖开
        "Big_Average": average_big,
        "Buy_Average_2": average_big_bid2,     # 全量top20里平均买大单量
        "Sell_Average_2": average_big_ask2     # 全量top20里平均卖大单量
    }
    # 上面写的时候忘round了，在这里统一round
    for k in list(result.keys()):
        if k in ["date", "symbol"]:
            continue
        v = result[k]
        result[k] = round(v, 2)
    for k in list(result2.keys()):
        if k in ["date", "symbol"]:
            continue
        result2[k] = round(result2[k], 2)
    return result, result2


def analyse_all_ask_bid():
    tick_data_path = "D:\自采tick"
    tick_temp_path = "d:\\temp"
    all_files = os.listdir(tick_data_path)
    all_files = all_files[:30]
    symbol_results = {}
    symbol_results2 = {}
    for file_name in all_files:
        print(file_name)
        tick_data_zip = zf.ZipFile(f"{tick_data_path}\\{file_name}", mode="r")
        tick_data_zip.extractall(tick_temp_path)
        all_csv_files = os.listdir(tick_temp_path)
        dt, _ = file_name.split(".")
        for csv_file_name in all_csv_files:
            print(file_name, csv_file_name)
            if "bar" in csv_file_name:
                continue
            if "99" in csv_file_name:
                continue
            if "&" in csv_file_name:
                continue
            _, sec_name = csv_file_name.split("_", 1)
            sec_id = extract_sec_id(sec_name)
            symbol, _ = sec_name.split(".", 1)
            try:
                df = pd.read_csv(f"{tick_temp_path}\\{csv_file_name}", encoding="GBK")
            except UnicodeDecodeError:
                df = pd.read_csv(f"{tick_temp_path}\\{csv_file_name}")
            result, result2 = analyse_ask_bid(dt, symbol, df)
            symbol_result = symbol_results.setdefault(symbol, [])
            symbol_result2 = symbol_results2.setdefault(symbol, [])
            if result:
                symbol_result.append(result)
            if result2:
                symbol_result2.append(result2)
        for csv_file_name in all_csv_files:
            os.remove(f"{tick_temp_path}\\{csv_file_name}")
    for symbol, symbol_data in symbol_results.items():
        symbol_df = pd.DataFrame(symbol_data)
        symbol_df.to_csv(f"d:\\tick_volume_ask_bid\\{symbol}.csv", index=False)
    for symbol, symbol_data in symbol_results2.items():
        symbol_df = pd.DataFrame(symbol_data)
        symbol_df.to_csv(f"d:\\tick_volume_ask_bid2\\{symbol}_2.csv", index=False)


if __name__ == "__main__":
    analyse_all_ask_bid()